package com.zhht.checkcop.demo;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Map.Entry;

import org.apache.log4j.Logger;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import com.zhht.checkcop.db.MysqlDb;
import com.zhht.checkcop.db.StaticDBMap;

/**
 * 掌阅公版书籍爬取
 * http://m.zhangyue.com/category/92/126?currentPage=215
 * @author love
 *
 */
public class ZhangYuePublicMenu implements PageProcessor{
	private final static Logger log= Logger.getLogger(ZhangYuePublicMenu.class);
	private Boolean pageURL=new Boolean(true);
	
	private Site site = Site.me().setCycleRetryTimes(5).setSleepTime(1500).setTimeOut(6000);
	private String id;
	private String sid;
	public ZhangYuePublicMenu(){}
	public ZhangYuePublicMenu(String id,String sid){
		this.id = id;
		this.sid=sid;
	}

	public void process(Page page) {
    	try{
//    		System.out.println(page);
//    		System.out.println(page.getHtml());
	    	List<String> display_name1=page.getHtml().xpath("//ul[@class='category_booklist']/li/a/@href").all();	
	    	List<String> display_name2=page.getHtml().xpath("//div[@class='category_book']/img/@data-src").all();	
	    	List<String> display_name3=page.getHtml().xpath("//dl[@class='book_info flex_item']/dt/text()").all();	
	    	List<String> display_name4=page.getHtml().xpath("//dl[@class='book_info flex_item']/dd[@class='author']/text()").all();	
	    	List<String> display_name5=page.getHtml().xpath("//dl[@class='book_info flex_item']/dd[@class='content ellipsis2']/text()").all();	
	    	
	    	for(int i=0;i<display_name1.size();i++){
	    	Map<String,Object> fieldsValue=new HashMap<String,Object>();
	    	String val1=display_name1.get(i);
	    	String val2=display_name2.get(i);
	    	String val3=display_name3.get(i);
	    	String val4=display_name4.get(i);
	    	String val5=display_name5.get(i);
	    	fieldsValue.put("name", val3);
	    	fieldsValue.put("author", val4);
	    	fieldsValue.put("cover_url", val2);
	    	fieldsValue.put("intro", val5);
	    	fieldsValue.put("zyId", val1);
	    	StaticDBMap.mysqldb.insert("t_zy_book", fieldsValue);
	    	}
		} catch (Exception e) {
			e.printStackTrace();
		}
    }
	
	/** 
     * 判断是否为中文字符 
     *  
     * @param c 
     * @return 
     */  
    public static boolean isChinese(char c) {  
        Character.UnicodeBlock ub = Character.UnicodeBlock.of(c);  
        if (ub == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS  
                || ub == Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS  
                || ub == Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A  
                || ub == Character.UnicodeBlock.GENERAL_PUNCTUATION  
                || ub == Character.UnicodeBlock.CJK_SYMBOLS_AND_PUNCTUATION  
                || ub == Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS) {  
            return true;  
        }  
        return false;  
    }  
	//Unicode转中文  
    public static String decodeUnicode(final String unicode) {     
         StringBuffer string = new StringBuffer();  
         
         String[] hex = unicode.split("\\\\u");  
   
         for (int i = 0; i < hex.length; i++) {  
               
             try {  
                 // 汉字范围 \u4e00-\u9fa5 (中文)  
                 if(hex[i].length()>=4){//取前四个，判断是否是汉字  
                     String chinese = hex[i].substring(0, 4);  
                     try {  
                         int chr = Integer.parseInt(chinese, 16);  
                         boolean isChinese = isChinese((char) chr);  
                         //转化成功，判断是否在  汉字范围内  
                         if (isChinese){//在汉字范围内  
                             // 追加成string  
                             string.append((char) chr);  
                             //并且追加  后面的字符  
                             String behindString = hex[i].substring(4);  
                             string.append(behindString);  
                         }else {  
                             string.append(hex[i]);  
                         }  
                     } catch (NumberFormatException e1) {  
                         string.append(hex[i]);  
                     }  
                       
                 }else{  
                     string.append(hex[i]);  
                 }  
             } catch (NumberFormatException e) {  
                 string.append(hex[i]);  
             }  
         }  
   
         return string.toString();  
    }  

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
    	try{
    		StaticDBMap.mysqldb = new MysqlDb();
    		StaticDBMap.mysqldb.connect();
    		Random r = new Random(1);
    		for(int i=1;i<=215;i++){
    			String url ="http://m.zhangyue.com/category/92/126?currentPage="+i;
    			Spider.create(new ZhangYuePublicMenu("",i+"")).addUrl(url).thread(1).run();
        		//Spider.create(new ZhangYuePublicMenu("2","946d8372884d445eb0a4cc6e3826dbe8")).addUrl("https://hanyu.baidu.com/shici/detail?pid=946d8372884d445eb0a4cc6e3826dbe8").thread(1).run();
        		int th = r.nextInt(1000);
				Thread.sleep(th);
    		}
    	}catch (Exception e) {
			e.printStackTrace();
		}
    	
    }
    
    private String subContent(String s,StringBuffer dlfy){
    	if(s.indexOf("<span") == -1){
    		return s;
    	}
    	String biao = s.substring(s.indexOf("data=")+6, s.indexOf("\">"));
    	dlfy.append(biao+"$$");
		String end = s.substring(s.indexOf("\">")+2);
		String nr = end.substring(0, end.indexOf("</span>"));
		String nrend = end.substring(end.indexOf("</span>")+7);
		return subContent("$$"+nr+"$$"+nrend,dlfy);
    }
    
	private static void saveAndUpdate() {
		try{
        	MysqlDb mysqldb = new MysqlDb();
        	mysqldb.connect();
        	if(null != StaticDBMap.writeNovelNameMap && StaticDBMap.writeNovelNameMap.size() > 0){
	        	//新增
	        	Iterator<Entry<String, Map<String,Object>>> iter = StaticDBMap.writeNovelNameMap.entrySet().iterator();
	        	List<String[]> sqls=new ArrayList<String[]>();
	        	while (iter.hasNext()) {
	        		Map.Entry<String, Map<String,Object>> entry = iter.next();
	        		sqls.add(mysqldb.assemblyInsertSqls("movie_info",entry.getValue()));
	            }
	        	mysqldb.executeBatchs(sqls);
        	}
        	mysqldb.close();
        }catch (Exception e) {
        	e.printStackTrace();
        }
	}
}
